/*
*************************************************
				Copyagent
	------------------------------------------
	            crawler.c
*************************************************
Fisierul principal. Contine rutinele MPI
*/


#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <errno.h>
#include <stdio.h>
#include <netdb.h>
#include <strings.h>
#include <string.h>
#include <unistd.h>
#include <mpi.h>
#include "crawler_lib.h"
#include "baza_date_lib.h"




extern int errno;
void verific_status(struct status_eroare status);
int waiting_size,aux_waiting_size;


int main (int argc, char **argv)
{
	int my_rank = 0;
	int ret = 0;
	int size = 0;
	int aux = 0;
	int minusuri = 0;
	int am_primit = 0;
	int sd; //socket descriptor
	struct hostent *hp;
	struct sockaddr_in server; //structura folosita pentru conectare
	int port = 80; //portul de comunicatie (80 - HTTP)
	char * maskFile = "urls.txt";
	char * urlFile = (char *)malloc(FILENAME_LENGTH);
	struct URLList * listaUrl = 
		(struct URLList *)malloc(sizeof(struct URLList));
	struct URLList * nod = (struct URLList *)malloc(sizeof(struct URLList));
	
	/*Folosit pentru a determina starea la intoarcerea din functie*/
	struct status_eroare status ;

	/*Numele fisierului inc are se va salva info de pe site*/
	char * url_file = (char *)malloc(FILENAME_LENGTH);
	
	char * buffer = (char*)malloc(BUFFER_SIZE);
	
	char * domeniu= (char *)malloc(FILENAME_LENGTH);
	char * cale_rel= (char *)malloc(FILENAME_LENGTH);


	/*bufferul de propozitii*/
	char * prop_buffer = (char *)malloc(BUFFER_SIZE);
	/*bufferul de link-uri */
	char * url_buffer= (char *)malloc(BUFFER_SIZE);
	/*host = www.ceva.ro*/
	char * host = (char *)malloc(FILENAME_LENGTH);
	/* Inititalizarea MPI*/
	ret= MPI_Init(&argc, &argv);
	
	ret= MPI_Comm_size( MPI_COMM_WORLD, &size );
	
	ret= MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
	
	
	waiting_size = size;

	/*Un scurt test*/
	printf("{%d}[%d of %d]\n",my_rank,my_rank, size);

	/*Fiecare nod are fisierul lui URL-uri*/
	sprintf(urlFile,"%d%s",my_rank,maskFile);

	/*Citesc fisierul de URL-uri si le pun in lista*/
#ifdef DEBUG
	printf("{%d} Citesc fisierul de configurare: %s\n", my_rank, urlFile);
#endif

	
	ret = readFile(urlFile,listaUrl);
	if (ret)
	{
		perror("Eroare la citirea fisierului de configurare\n");
		MPI_Finalize();
		return errno;
	}
	

	nod = listaUrl;

	if (nod == NULL)
	{
		/*Lista URL este goala*/
		MPI_Finalize();
		return 0;
		
	}
ret = mysql_connection_start();
if (ret == 0)
while(nod)
{
		printf("{%d}[While %s]\n",my_rank,nod->url);
		fflush(stdout);
	

	nod->parcurs = ADEVARAT;
	// crearea socketului
	sd = socket(AF_INET, SOCK_STREAM,0);
	if (sd == -1)
	{
		perror("Eroare la creare socket\n");
		nod = nod -> next;
		continue;
	}
	
	
	url_file = normalizez_nume_fisier(nod->url);
	
	replaceChars(url_file,NULL,'_');

	if (url_file == NULL)
	{
		nod = nod -> next;
		continue;
	}
	if (strlen(url_file) < MIN_URL_LEN )
	{
		nod = nod -> next;
		continue;
	}
	
	

	host = extrageHostDinURL(nod->url);

	
	hp = gethostbyname(host);
		

	if( hp == NULL )
	{

		perror("Eroare la gethostbyname\n");
		nod = nod -> next;
		continue;
	

  	}

	//umplem structura folosita pentru realizarea conexiunii cu serverul
	bcopy(hp->h_addr, &server.sin_addr, hp->h_length);
	server.sin_family = AF_INET;
	server.sin_port = htons(port);
	
#ifdef DEBUG
	printf("{%d}[Connect...%s:%d]\n",my_rank,host,server.sin_port);
	fflush(stdout);	
#endif	
	
	ret = connect ( sd, (struct sockaddr *) &server, sizeof (struct sockaddr) );
	
	if ( ret == -1 )
	{

		perror("Eroare la connect()");
		close(sd);
		nod = nod -> next;
		continue;

	} 
	
	memset(domeniu, 0, FILENAME_LENGTH);
	status = relativ_din_absolut(nod->url , domeniu, cale_rel);
	verific_status(status);

	printf("URL: %s | dom: %s | rel: %s \n",nod->url, domeniu, cale_rel);
	fflush(stdout);	


	/*Cream HTTP request*/
	memset(buffer,0,BUFFER_SIZE);
	sprintf(buffer,"GET %s HTTP/1.1\n", cale_rel);

	printf("{%d}[Scriu...][%s]\n",my_rank,buffer);
	

	
	ret = write(sd, buffer,strlen(buffer));
	
	if (ret <= 0 )
	{
		perror("Eroare la trimitere GET");
		close(sd);
		nod = nod -> next;
		continue;
	}
	
	memset(buffer,0,BUFFER_SIZE);
	sprintf(buffer,"Host: %s\n\n",domeniu);

	
	printf("{%d}[Scriu...][%s]\n", my_rank, buffer);
	fflush(stdout);	


	ret = write(sd, buffer,strlen(buffer));
	if (ret <= 0 )
	{
		perror("Eroare la trimitere Host");
		close(sd);
		nod = nod -> next;
		continue;
	}
	

	printf("{%d}[Am scris...]\n", my_rank);
	fflush(stdout);	

	
ret = 1;
	
			
	while ( ( ret != 0 ) )
	{
		aux++;
		memset(buffer,0,BUFFER_SIZE);
		ret = read(sd, buffer, BUFFER_SIZE);
		if ( ret == -1)
		{
			printf( "{%d} Eroare la citirea de pe socket\n", my_rank);
			fflush(stdout);
			break;
		}
		/*Pot aparea probleme cand  dupa  simbolul % apare un simbol pe care fprintf incearca sa-l interpreteze ca in %s, %d, etc... 
		fprintf ignora spatiile dintre % si aceste litere*/
		replaceChars(buffer,'%','_');
		parse_buffer(buffer, url_buffer, prop_buffer);

		sparge_prop_bufferul_pentru_bd(prop_buffer, nod->url);
		sparge_link_bufferul(url_buffer, nod->url, listaUrl);
		
			
	}

	close(sd);
	//fclose(stream);
	//fclose(link_file);
	//fclose(prop_file);
	
	

	am_primit = schimb_de_linkuri( nod , listaUrl);
	printf( "{%d} schimb_de_linkuri %d\n", my_rank, am_primit);
	fflush(stdout);	
	
	if (am_primit == FALS)
		minusuri = 1;
	if (waiting_size > 1)
	{
		MPI_Allreduce(&minusuri,&aux_waiting_size,1, MPI_INT,MPI_SUM, MPI_COMM_WORLD);
		waiting_size -= aux_waiting_size;
		
		printf("{%d}_Waiting: %d\n", my_rank, waiting_size);
		fflush(stdout);
	}
	printf( "{%d} Gata %s\n", my_rank, nod->url);
	fflush(stdout);	
	
	nod = nod->next;
}

	free(buffer);



	
	
	


	printf("{%d} Am terminat. \n", my_rank);

	mysql_connection_end();
	MPI_Finalize();
	return 0;

}

/*
void schimb_de_linkuri( struct URLList * nod, struct URLList * lista);
-------------------------------------------------
Inainte de a merge la urmatorul link din lista verificam daca acesta exista.
Daca nu exista, le vom ruga :-) pe nodurile vecine sa ne ajute.
-------------------------------------------------
return int: 1 daca am primit ce am avut nevoie, 0 altfel
struct URLList * nod: nodul curent
struct URLList * listaUrl: lista din care face parte si nodul curent
*/
int schimb_de_linkuri( struct URLList * nod, struct URLList * listaUrl)
{
	int need_url = 0,my_rank = 0, primire = ADEVARAT;
	MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

	if (nod->next == NULL)
	{
		need_url = ADEVARAT;
	}
	else
	{
		need_url = FALS;
	}
	
	int * noduri_fara_url = (int *)(malloc(BIOLUP_SIZE*sizeof(int)));
	memset (noduri_fara_url, 0, BIOLUP_SIZE * sizeof(int));

	
	int message[BIOLUP_SIZE+1];
	memset (message, 0, (BIOLUP_SIZE+1) * sizeof(int));

	apel_lipsa_linkuri(need_url, listaUrl, noduri_fara_url);
	
	broadcast_cerere_linkuri(noduri_fara_url, message);
	
	primire = trimitere_linkuri(message, listaUrl);
	
	return primire;
}
/*
int message_contains(int*message, int size);
-------------------------------------------------
Verifica daca my_rank se regaseste intr-un vector de dimensiune size
-------------------------------------------------
return int: ADEVARAT sau FALS;
int size: dimensiunea vectorului
int * message: vectorul in care se cauta

*/
int message_contains(int * message, int size )
{
	int i = 0, my_rank = 0;
	MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
	for( i = 0 ; i <= size ; i++ )
		printf(" %d |", message[i]);
	printf("\n rank: %d\n", my_rank);
	
	for (i = 1 ; i <= size ; i++)
		if (message[i] == my_rank)
		{
			return ADEVARAT;
		}
		
	return FALS;
}





/*
void trimitere_linkuri(int * message);
-------------------------------------------------
Fiecare nod incearca sa ajute nodurile care au ramas fara link-uri
-------------------------------------------------
return int: 1 daca am primit ce am avut nevoie, 0 altfel
int * message: mesajul care contine nodurile care au nevoie de ajutor
struct URLList * lista: lista din care se iau link-urile
*/
int trimitere_linkuri(int * message, struct URLList * lista)
{
		int i = 0, j = 0, my_rank = 0, no_url_count = 0, noduri_fara_url_size = 0, primire=1;
		int donor_size = 0, acceptor_size = 0 , sunt_acceptor = 0;
		MPI_Status status;
		char * url= (char *)malloc(FILENAME_LENGTH);	
		char * url_aux= (char *)malloc(FILENAME_LENGTH);	


		sprintf(url_aux,"gol");
		MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);

		noduri_fara_url_size = message[0];
		if (noduri_fara_url_size == 0)
		{
			return;
		}
		if (noduri_fara_url_size == waiting_size)
		{
			printf("{%d} Am ramas fara link-uri. \n", my_rank);
			fflush(stdout);
			return FALS;
		}
	

	
		acceptor_size = noduri_fara_url_size;
		donor_size = waiting_size - acceptor_size;

		
		

		sunt_acceptor = message_contains(message,noduri_fara_url_size);

		if (donor_size == 0)
			return FALS;

		if (sunt_acceptor == ADEVARAT)
		{
			printf("{%d}Astept sa primesc\n",my_rank);
			fflush(stdout);
			/*Sunt acceptor*/
			for ( i = 0 ; i < donor_size ; i++)
				{
					MPI_Recv(url, FILENAME_LENGTH, MPI_CHAR, MPI_ANY_SOURCE, TAG_URL, MPI_COMM_WORLD, &status);
					if ( (url != NULL)&&(strcmp(url,"gol") != 0) )
					{	
						
						insertNode(url, lista);
						printf("{%d}Am primit %s de la %d\n", my_rank, url, status.MPI_SOURCE);
						fflush(stdout);
					
					}
					else
					{
						printf("{%d}Am primit NULL de la %d\n", my_rank,  status.MPI_SOURCE);
						fflush(stdout);
						no_url_count++;
					}

				}
			if ( no_url_count == donor_size)
				primire = FALS;

			
		}
		else
		{
			/*Sunt donor*/
			for ( j = 1 ; j <= noduri_fara_url_size ; j++)	
			{
			
					url = scoate_penultimul_url(lista);
					if (url == NULL)
						url = url_aux;

					printf("{%d}Trimit %s lui %d\n", my_rank, url, message[j]);
					fflush(stdout);
					
					MPI_Send(url, FILENAME_LENGTH, MPI_CHAR, message[j], TAG_URL, MPI_COMM_WORLD);	
				
			}

		}
		
						
	return primire;


}

/*
broadcast_cerere_linkuri(int * noduri_fara_url, int * message);
-------------------------------------------------
Nodul radacina va face un broadcast cu un mesaj in care spune
cine are nevoie de link-uri
-------------------------------------------------
int * noduri_fara_url:	un vector de dimensiune BIOLUP_SIZE care
						contine pe pozitia NOD nevoia de link: ADEVARAT/FALS
int * message: mesajul din care nodurile vor descifra cui sa trimita eventualele linkuri
*/
void broadcast_cerere_linkuri(int * noduri_fara_url, int * message)
{
	int i = 0, j = 0, my_rank = 0, noduri_fara_url_size = 0;

	MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
	
	if (my_rank == 0)
	{
		for( i = 0 ; i < BIOLUP_SIZE ; i++ )
			if ( noduri_fara_url[i] == 1)
				message[++j] = i;
		
		noduri_fara_url_size = j ;	
		message[0] = noduri_fara_url_size;
		
	}
	

	
	MPI_Bcast(message, waiting_size + 1, MPI_INT, 0, MPI_COMM_WORLD);
		
}



/*
int *  apel_lipsa_linkuri(short need_url, struct URLList * lista);
-------------------------------------------------
La sfarsitul prelucrarii unui url, inainte de trecerea la url-ul urmator
se verifica daca acesta exista. Daca nu, se transmite un mesaj la nodul 0, 
acesta va face un broadcast in care ii va anunta pe toti ca nodul X are nevoie
de link-uri. Fiecare nod va extrage din lista ultimul sau url daca poate si il
va trimite nodului X
-------------------------------------------------
return noduri_fara_url: vectorul care arata cine are nevoie de link-uri
short need_link: am nevoie de link ?
struct URLList * lista: lista in care se vor adauga link-urile daca vor veni
int * noduri_fara_url: vector de BIOLUP_SIZE dimensiune. Indexul este numarul nodului. 
	Pe fiecare pozitie va fi 1 daca nodul are nevoie de link si 0 altfel.
*/
int * apel_lipsa_linkuri(int need_url, struct URLList * lista, int * noduri_fara_url)
{

	int i = 0,  my_rank = 0;

	MPI_Status status;
	char * buffer_sos = (char *)malloc(FILENAME_LENGTH);
	
	MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
	
	sprintf(buffer_sos, "%d", need_url);
	
	/*
		Toata lumea trimite mesaj root-ului. Daca mesajul este 0 atunci inseamna ca nu are nevoie
	de url. Daca este 1 inseamna ca solicita URL
	*/
	if (my_rank == 0)
	{
		for (i = 1 ; i < waiting_size ; i++)
		{

			MPI_Recv(buffer_sos, 1, MPI_CHAR, MPI_ANY_SOURCE, TAG_NEED_URL, MPI_COMM_WORLD, &status);
			if (buffer_sos[0] == '1')
			{
				noduri_fara_url[status.MPI_SOURCE] = 1;
			}
			else
			{
				noduri_fara_url[status.MPI_SOURCE] = 0;
			}
		}
		if(need_url == 1)
			noduri_fara_url[0] = 1;
	}
	else
	{
		MPI_Send(buffer_sos,1,MPI_CHAR,0, TAG_NEED_URL, MPI_COMM_WORLD);
	}


	return noduri_fara_url;
}

void verific_status(struct status_eroare status)
{
	if ( status.eroare == ADEVARAT)
	{
		printf("[ERR] %s\n",status.mesaj);
		
	}


}

